#!/bin/bash
## 运行脚本前请仔细阅读wiki(https://github.com/ymcui/Chinese-LLaMA-Alpaca-3/wiki/sft_scripts_zh)
## Read the wiki(https://github.com/ymcui/Chinese-LLaMA-Alpaca-3/wiki/sft_scripts_en) carefully before running the script

# torchrun --nnodes 1 --nproc_per_node 1 run_clm_sft_with_peft.py \
python G:/LLM/lover_llama/scripts/training/run_clm_sft_with_peft.py 
    --model_name_or_path G:/LLM/LLAMA3/origin_models/Qwen2-1.5B
    --tokenizer_name_or_path G:/LLM/LLAMA3/origin_models/Qwen2-1.5B
    --dataset_dir G:/LLM/lover_llama/datasets/train 
    --per_device_train_batch_size 4 
    --do_train 1
    --low_cpu_mem_usage True
    --seed 42
    --num_train_epochs 3 
    --lr_scheduler_type cosine 
    --learning_rate 1e-4 
    --warmup_ratio 0.03 
    --weight_decay 0.1 
    --logging_strategy steps 
    --logging_steps 10 
    --save_strategy steps 
    --save_total_limit 3 
    --evaluation_strategy steps 
    --eval_steps 100 
    --save_steps 200 
    --gradient_accumulation_steps 8 
    --preprocessing_num_workers 8 
    --max_seq_length 512 
    --output_dir G:/LLM/lover_llama/scripts/training/output 
    --overwrite_output_dir 1
    --ddp_timeout 30000 
    --logging_first_step True 
    --lora_rank 64 
    --lora_alpha 128 
    --trainable "q_proj,v_proj" 
    --lora_dropout 0.05 
    --validation_file G:/LLM/lover_llama/datasets/eval/ruozhiba_qa2449_gpt4o.json 
    --ddp_find_unused_parameters False
    --load_in_kbits 4
    --modules_to_save None


--per_device_eval_batch_size 8 
--trainable "q_proj,v_proj,k_proj,o_proj,gate_proj,down_proj,up_proj" 
--bf16 False
--do_eval 1
--use_flash_attention_2